#### （一）系统配置

# 固定 IP地址
ip a

# 安装ssh服务，并开启
apt install openssh-server -y
systemctl enable --now sshd

# 配置/etc/hosts 文件，并拷贝公钥
cat << EOF >> /etc/hosts
192.168.103.31 BI-01
192.168.103.32 BI-02
192.168.103.33 BI-03
EOF

ssh-keygen

ssh-copy-id BI-01
ssh-copy-id BI-02
ssh-copy-id BI-03

# 关闭swap分区
swapoff -a

# 关闭开机挂载swap分区
vi /etc/fstab 注释 swap

# 关闭开机启动防火墙
systemctl disable --now ufw

# 设置系统时区未 shanghai
timedatectl set-timezone Asia/Shanghai

# 重启时间同步服务
systemctl restart systemd-timesyncd.service

# 时间检查
timedatectl status

# 开启k8s ip转发
cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
EOF

# 检查是否生效
sudo sysctl --system

#### （二）安装 containerd

apt policy containerd
apt update && apt install -y containerd

# 检查containerd 版本(1.7.12)
containerd --version
dpkg -l | grep containerd

# 手动生成配置文件
sudo mkdir -p /etc/containerd && \
sudo containerd config default > /etc/containerd/config.toml

#修改/etc/containerd/config.toml 文件中：[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] 下 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] 下的 SystemdCgroup 为 true
vim /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
# 设置sandbox_image（避免后续报错）
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"

# 重启containerd服务
systemctl restart containerd

#### （三）安装 k8s 组件

# 安装必要的工具
sudo apt install -y apt-transport-https ca-certificates curl gpg

# 创建GPG key目录
mkdir -p -m 755 /etc/apt/keyrings

# 下载 Kubernetes 仓库的 GPG 密钥，并将其安装到 APT 的密钥环中，以便验证从该仓库下载的软件包
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | \
sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg && \
sudo chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg

# 添加 k8s 的apt仓库
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' \
| sudo tee /etc/apt/sources.list.d/kubernetes.list

# 下载 k8s，并标记软件包为 hold 状态，不被 apt 自动更新
sudo apt update && \
sudo apt install -y kubelet kubectl kubeadm && \
sudo apt-mark hold kubelet kubeadm kubectl

# 检查版本
kubelet --version && kubeadm version

# 开机启动 kubelet
sudo systemctl enable --now kubelet

# 提前下载所需要的组件镜像
sudo kubeadm config images pull --kubernetes-version=v1.30.1 --cri-socket=unix:///run/containerd/containerd.sock --image-repository=registry.aliyuncs.com/google_containers

# 初始化主节点(二选一)
sudo kubeadm init \
--apiserver-advertise-address=192.168.103.31 \
--control-plane-endpoint=bi-01 \
--kubernetes-version=v1.30.1 \
--service-cidr=10.50.0.0/16 \
--pod-network-cidr=10.60.0.0/16 \
--cri-socket=unix:///run/containerd/containerd.sock \
--image-repository=registry.aliyuncs.com/google_containers
# 或者
kubeadm init --config resources/kubeadm-config.yaml


#### calico网络组件安装

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config


wget https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml
# 或者
curl -O https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml

kubectl create -f resources/tigera-operator.yaml

# calico的配置文件，需要修改 pod ip address
curl -O https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/custom-resources.yaml
kubectl apply -f resources/custom-resources.yaml


#### 其他节点加入集群
kubeadm join bi-01:6443 --token ndrfzx.mgg77ltlcn9j68z1 \
        --discovery-token-ca-cert-hash sha256:3d698fd63ba956b706d356cd4dc21cb410e6217dcaa53b43caa98bc0469ba995 \
        --control-plane

# token过期重新生成
kubeadm token create

# ca hash查看
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null \
| sha256sum | awk '{print $1}'


#### 主节点开启调度，删除禁止调度污点
kubectl taint nodes k8s-master node-role.kubernetes.io/control-plane-

#### 安装 nginx-ingress
curl -O https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.10.1/deploy/static/provider/cloud/deploy.yaml

# 修改 externalTrafficPolicy: Cluster && type: NodePort
kubectl apply -f ingress-nginx/deploy.yaml

#### 失败重置选项
# ctr --namespace k8s.io image pull registry.aliyuncs.com/google_containers/pause:3.9
# ctr --namespace k8s.io image tag registry.aliyuncs.com/google_containers/pause:3.9 registry.k8s.io/pause:3.8

sudo kubeadm reset # 重置 kubeadm ，执行这个后需要敲 y 回车
sudo rm -rf /etc/cni/net.d # 删除上次 init 生成的文件
sudo rm -rf /var/lib/etcd # 删除上次 init 生成的文件

sudo rm -rf /etc/cni/net.d
sudo rm -rf /etc/kubernetes/*
sudo rm -rf ~/.kube/*
sudo rm -rf /var/lib/etcd/*
sudo rm -rf $HOME/.kube

kubectl drain <节点名称> --ignore-daemonsets --delete-emptydir-data # 驱逐节点上的所有 Pod
kubectl delete node <节点名称> # 从集群中删除节点